shared source/sscli20/samples/compilers/myc/src/tok.cs

   1 //------------------------------------------------------------------------------
   2 // <copyright file="tok.cs" company="Microsoft">
   3 //
   4 //      Copyright (c) 2006 Microsoft Corporation.  All rights reserved.
   5 //
   6 //      The use and distribution terms for this software are contained in the file
   7 //      named license.txt, which can be found in the root of this distribution.
   8 //      By using this software in any fashion, you are agreeing to be bound by the
   9 //      terms of this license.
  10 //
  11 //      You must not remove this notice, or any other, from this software.
  12 //
  13 // </copyright>
  14 //------------------------------------------------------------------------------
  15
  16 namespace MyC
  17 {
  18 using System;
  19 using System.Text;
  20 using System.Collections;
  21
  22 class Tok
  23   {
  24   public const int T_LEFT_ASSIGN        = 10001;
  25   public const int T_RIGHT_ASSIGN       = 10002;
  26   public const int T_LEFT_OP            = 10003;
  27   public const int T_LE_OP              = 10004;
  28   public const int T_RIGHT_OP           = 10005;
  29   public const int T_GE_OP              = 10006;
  30   public const int T_EQ_OP              = 10007;
  31   public const int T_NEQ_OP             = 10008;
  32   public const int T_INC_OP             = 10009;
  33   public const int T_ADD_ASSIGN         = 10010;
  34   public const int T_DEC_OP             = 10011;
  35   public const int T_SUB_ASSIGN         = 10012;
  36   public const int T_PTR_OP             = 10013;
  37   public const int T_MUL_ASSIGN         = 10014;
  38   public const int T_DIV_ASSIGN         = 10015;
  39   public const int T_DIV_OP             = 10016;
  40   public const int T_LOG_OR_OP          = 10017;
  41   public const int T_LOG_AND_OP         = 10018;
  42   public const int T_LT_OP              = 10019;
  43   public const int T_GT_OP              = 10020;
  44   public const int T_ASSIGN             = 10021;
  45   public const int T_ADD_OP             = 10022;
  46   public const int T_SUB_OP             = 10023;
  47   public const int T_MUL_OP             = 10024;
  48   public const int T_OR_OP              = 10025;
  49   public const int T_AND_OP             = 10026;
  50   public const int T_MOD_OP             = 10027;
  51   public const int T_XOR_OP             = 10028;
  52   public const int T_COMPL_OP           = 10029;
  53   public const int T_NOT_OP             = 10030;
  54
  55   public const int T_IF                 = 20001;
  56   public const int T_ELSE               = 20002;
  57   public const int T_WHILE              = 20003;
  58   public const int T_FOR                = 20004;
  59   public const int T_DO                 = 20005;
  60   public const int T_BREAK              = 20006;
  61   public const int T_CONTINUE           = 20007;
  62   public const int T_RETURN             = 20008;
  63   public const int T_FLOW_MIN           = T_IF;
  64   public const int T_FLOW_MAX           = T_RETURN;
  65
  66   public const int T_EXTERN             = 30001;
  67   public const int T_STATIC             = 30002;
  68   public const int T_AUTO               = 30003;
  69   public const int T_SIGNED             = 30004;
  70   public const int T_UNSIGNED           = 30005;
  71   public const int T_DEFCLASS           = 30006;
  72   public const int T_STORAGE_MIN        = T_EXTERN;
  73   public const int T_STORAGE_MAX        = T_DEFCLASS;
  74   public const int T_PARAM              = 30007 /* special for tagging parameters */;
  75
  76   public const int T_SHORT              = 40001;
  77   public const int T_CHAR               = 40002;
  78   public const int T_INT                = 40003;
  79   public const int T_LONG               = 40004;
  80   public const int T_FLOAT              = 40005;
  81   public const int T_DOUBLE             = 40006;
  82   public const int T_VOID               = 40007;
  83   public const int T_DEFTYPE            = 40008;
  84   public const int T_TYPE_MIN           = T_SHORT;
  85   public const int T_TYPE_MAX           = T_DEFTYPE;
  86
  87   public const int T_IDENT              = 50001;
  88   public const int T_DIGITS             = 50002;
  89   public const int T_UNKNOWN            = 99999;
  90   public const int T_EOF                = -1;
  91
  92   static Hashtable tokens;
  93   StringBuilder value;
  94   int token_id;
  95   Io io;
  96
  97   public static void AddTok(int i, String s)
  98     {
  99     tokens.Add(s, i);
 100     tokens.Add(i, s);
 101     }
 102
 103   public void InitHash()
 104     {
 105     tokens = new Hashtable();
 106     AddTok(T_LEFT_ASSIGN,       "<<=");
 107     AddTok(T_RIGHT_ASSIGN,      ">>=");
 108     AddTok(T_LEFT_OP,           "<<");
 109     AddTok(T_LE_OP,             "<=");
 110     AddTok(T_RIGHT_OP,          ">>");
 111     AddTok(T_GE_OP,             ">=");
 112     AddTok(T_EQ_OP,             "==");
 113     AddTok(T_NEQ_OP,            "!=");
 114     AddTok(T_INC_OP,            "++");
 115     AddTok(T_ADD_ASSIGN,        "+=");
 116     AddTok(T_DEC_OP,            "--");
 117     AddTok(T_SUB_ASSIGN,        "-=");
 118     AddTok(T_PTR_OP,            "->");
 119     AddTok(T_MUL_ASSIGN,        "*=");
 120     AddTok(T_DIV_ASSIGN,        "/=");
 121     AddTok(T_DIV_OP,            "/");
 122     AddTok(T_LOG_OR_OP,         "||");
 123     AddTok(T_LOG_AND_OP,        "&&");
 124     AddTok(T_LT_OP,             "<");
 125     AddTok(T_GT_OP,             ">");
 126     AddTok(T_ASSIGN,            "=");
 127     AddTok(T_ADD_OP,            "+");
 128     AddTok(T_SUB_OP,            "-");
 129     AddTok(T_MUL_OP,            "*");
 130     AddTok(T_OR_OP,             "|");
 131     AddTok(T_AND_OP,            "&");           /* could be addr of, or bitwise and */
 132     AddTok(T_MOD_OP,            "%");
 133     AddTok(T_XOR_OP,            "^");
 134     AddTok(T_COMPL_OP,          "~");
 135     AddTok(T_NOT_OP,            "!");
 136
 137     AddTok(T_IF,                "if");
 138     AddTok(T_ELSE,              "else");
 139     AddTok(T_WHILE,             "while");
 140     AddTok(T_FOR,               "for");
 141     AddTok(T_DO,                "do");
 142     AddTok(T_BREAK,             "break");
 143     AddTok(T_CONTINUE,          "continue");
 144     AddTok(T_RETURN,            "return");
 145
 146     AddTok(T_EXTERN,            "extern");
 147     AddTok(T_STATIC,            "static");
 148     AddTok(T_AUTO,              "auto");
 149     AddTok(T_SIGNED,            "signed");
 150     AddTok(T_UNSIGNED,          "unsigned");
 151
 152     AddTok(T_INT,               "int");
 153     AddTok(T_LONG,              "long");
 154     AddTok(T_CHAR,              "char");
 155     AddTok(T_FLOAT,             "float");
 156     AddTok(T_DOUBLE,            "double");
 157     AddTok(T_VOID,              "void");
 158     }
 159
 160   public Tok(Io ihandle)
 161     {
 162     io = ihandle;
 163     InitHash();                 // initialize the tokens hashtable
 164     io.ReadChar();
 165     scan();
 166     }
 167
 168   /*
 169    * table lookup
 170    *
 171    * if the input string matches a table entry, return the entry token_id
 172    * if not, return -1.
 173    */
 174   int lookup_id()
 175     {
 176     String s = value.ToString();
 177     Object k = tokens[s];
 178     if (k == null)
 179       return 0;
 180     return (int) k;
 181     }
 182
 183   /* constant declaration */
 184   const char TAB = '\t';
 185   const char CR = '\r';
 186   const char LF = '\n';
 187
 188   /* recognize any operator */
 189   bool isOp(char c)
 190     {
 191     return (c == '+' || c == '-' || c == '*' || c == '/' ||
 192             c == '<' || c == '>' || c == '=' ||
 193             c == '&' || c == '|' || c == '^' || c == '!'
 194             );
 195     }
 196
 197   bool isAddOp(char c)
 198     {
 199     return ((c == '+' || c == '-'));
 200     }
 201
 202   bool isMulOp(char c)
 203     {
 204     return ((c == '*' || c == '/'));
 205     }
 206
 207   bool isOrOp(char c)
 208     {
 209     return ((c == '|') || (c == '~'));
 210     }
 211
 212   bool isRelOp(char c)
 213     {
 214     return ((c == '=') || (c == '!') || (c == '<') || (c == '>'));
 215     }
 216
 217   void skipWhite()
 218     {
 219     while (Char.IsWhiteSpace(io.getNextChar()))
 220       io.ReadChar();
 221     }
 222
 223 /* get an identifier */
 224 void LoadName()
 225   {
 226   value = new StringBuilder(MyC.MAXSTR);
 227   skipWhite();
 228   if (!Char.IsLetter(io.getNextChar()))
 229     throw new ApplicationException("?Expected Name");
 230   while (Char.IsLetterOrDigit(io.getNextChar()))
 231     {
 232     value.Append(io.getNextChar());
 233     io.ReadChar();
 234     }
 235   token_id = lookup_id();
 236   if (token_id <= 0)
 237     token_id = T_IDENT;
 238   skipWhite();
 239   }
 240
 241 /* get a number */
 242 void LoadNum()
 243   {
 244   value = new StringBuilder(MyC.MAXSTR);
 245   skipWhite();
 246   if (!Char.IsDigit(io.getNextChar()))
 247     throw new ApplicationException("?Expected Integer");
 248   while (Char.IsDigit(io.getNextChar()))
 249     {
 250     value.Append(io.getNextChar());
 251     io.ReadChar();
 252     }
 253   token_id = T_DIGITS;
 254   skipWhite();
 255   }
 256
 257 /* get an operator */
 258 void LoadOp()
 259   {
 260   value = new StringBuilder(MyC.MAXSTR);
 261   skipWhite();
 262   if (!isOp(io.getNextChar()))
 263     throw new ApplicationException("?Expected operator");
 264   while (isOp(io.getNextChar()))
 265     {
 266     value.Append(io.getNextChar());
 267     io.ReadChar();
 268     }
 269   token_id = lookup_id();
 270   skipWhite();
 271   }
 272
 273 /* get an identifier */
 274   public void scan()
 275     {
 276     skipWhite();
 277     if (Char.IsLetter(io.getNextChar()))
 278       LoadName();
 279     else if (Char.IsDigit(io.getNextChar()))
 280       LoadNum();
 281     else if (isOp(io.getNextChar()))
 282       LoadOp();
 283     else if (io.EOF())
 284       {
 285       value = null;
 286       token_id = T_EOF;
 287       }
 288     else
 289       {
 290       value = new StringBuilder(MyC.MAXSTR);
 291       value.Append(io.getNextChar());
 292       token_id = T_UNKNOWN;
 293       io.ReadChar();
 294       }
 295     skipWhite();
 296 #if DEBUG
 297     Console.WriteLine("[tok.scan tok=["+this+"]");
 298 #endif
 299     }
 300
 301   public char getFirstChar()
 302     {
 303     if (value == null)
 304       return '\0';
 305     return value[0];
 306     }
 307
 308   public String getValue()
 309     {
 310     if (value == null)
 311       return "";
 312     return value.ToString();
 313     }
 314
 315   public int getId()
 316     {
 317     return (token_id);
 318     }
 319
 320   public bool NotEOF()
 321     {
 322     return (token_id != T_EOF);
 323     }
 324
 325   public bool IsDeclKeyword()
 326     {
 327     if ((token_id >= T_TYPE_MIN && token_id <= T_TYPE_MAX) ||
 328         (token_id >= T_STORAGE_MIN && token_id <= T_STORAGE_MAX))
 329       return true;
 330     return false;
 331     }
 332
 333   public override string ToString()
 334     {
 335     StringBuilder sb = new StringBuilder(getValue());
 336     sb.Append("(");
 337     sb.Append(token_id);
 338     sb.Append(")");
 339     return sb.ToString();
 340     }
 341
 342   }
 343 }